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Multi-Processor Architecture 


NGINX Configuration (Instance 1) 


user root; 

worker. processes 48 ; 

worker. cpu affinity auto 
000000000000000000000000111111111111111111111111000000000000000000000000111111111111111111111111; 

worker. rlimit nofile 1024000; 

error log /home/ubuntu/access.error error; 


NGINX Configuration (Instance 2) 


user root; 

worker. processes 48 ; 

worker. cpu. affinity auto 
111111111111111111111111000000000000000000000000111111111111111111111111000000000000000000000000 ; 

worker. rlimit nofile 1024000; 

error. log /home/ubuntu/access.error error; 


Deploying NGINX Instances 


nginx -c /path/to/configuration/instance-1 
nginx -c /path/to/configuration/instance-2 
ps aux | grep nginx 
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nginx: master process /usr/sbin/nginx -c /etc/nginx/nginx. 0.conf 
nginx: worker process 
nginx: worker process 


nginx: master process /usr/sbin/nginx -c /etc/nginx/nginx 1.conf 
nginx: worker process 


nginx: worker process 


$ pkill nginx 
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Web Server (Instance 1) 


events ( 
worker. connections 1000000; 
} 
http ( 
access. log off; 
keepalive timeout 315; 
keepalive requests 10000000; 
sendfile on; 
tcp. nopush on; 
tcp. nodelay on; 
server ( 
listen 10.10.16.10:443 backlog-250000 reuseport; 
root /usr/share/nginx/bin; 
} 


Web Server (Instance 2) 


events ( 
worker. connections 1000000; 
} 
http ( 
access. log off; 
keepalive timeout 315; 
keepalive requests 10000000; 
sendfile on; 
tcp. nopush on; 
tcp. nodelay on; 
server ( 
listen 10.10.11.23:443 backlog-250000 reuseport; 
root /usr/share/nginx/bin; 
} 
} 


Reverse Proxy (Instance 1) 


server ( 
listen 10.10.10.18:443 ssl backlog-102400 reuseport; 
ssl certificate /etc/ssl/certs/nginx.pem; 
ssl certificate key /etc/ssl/private/nginx.key; 
ssl session cache off; 
551 session tickets off: 


location / ( 
proxy http. version 1.1; 
proxy.set header Connection ""; 
proxy pass http://webserver 0; 


} 
} 
upstream webserver 8 4 
server 10.10.10.11:80; 
keepalive 200; 
) 
} 
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Reverse Proxy (Instance 2) 


server ( 
listen 10.10.15.9:443 ssl backlog-102400 reuseport; 
ssl certificate /etc/ssl/certs/nginx.pem; 
ssl certificate key /etc/ssl/private/nginx.key; 
ssl session cache off; 
ssl session tickets off; 


location / ( 
proxy http. version 1.1; 
proxy.set header Connection ""; 
proxy pass http://webserver. 1; 
} 
} 
upstream webserver 1 4 
server 10.10.15.12:80; 
keepalive 200; 
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Web Server (Instance 1) 


events ( 
worker. connections 1000000; 
} 
http ( 
access. log off; 
keepalive timeout 315; 
keepalive requests 10000000; 
sendfile on; 
tcp. nopush on; 
tcp. nodelay on; 
server ( 
listen 10.10.16.10:443 backlog-250000 reuseport; 
root /usr/share/nginx/bin; 
} 


Web Server (Instance 2) 


events ( 
worker. connections 1000000; 
} 
http ( 
access. log off; 
keepalive timeout 315; 
keepalive requests 10000000; 
sendfile on; 
tcp. nopush on; 
tcp. nodelay on; 
server ( 
listen 10.10.11.23:443 backlog-250000 reuseport; 
root /usr/share/nginx/bin2; 
} 
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Performance Test Results 
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Linux Sysctl Settings 


* Increase memory thresholds to prevent packet dropping 
o sysctl -w net.ipv4.tcp_rmem="4096 87380 4194304" 
o sysctl -w net.ipv4.tcp_wmem="4096 65536 4194304" 
* Increase the size of the processor queues 
° sysctl -w net.core.net dev max backlog-250000 
e Setting the maximum TCP buffer sizes 
o sysctl -w net.core.. mem max-4194304 
o sysctl -w net.core.wmem_max=41 94304 
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Linux Sysctl Settings 


• Disable TCP timestamps 
sysctl -w net.ipv4.tcp timestamps-0 
• Defines the local port range that is used by TCP and UDP to 
choose the local port 
sysctl -w net.ipv4.ip. local port range = 32768 60999 
e Enable reuse of TIME-WAIT sockets for new connections when 
itis safe from protocol viewpoint. 
sysctl -w net.ipv4.tcp. tw reuse = 1 
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Linux Sysctl Settings 
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Enabling RSS and TPS 


e /etc/init.d/irqg balance stop 

e git clone https://github.com/ANLAB-KAIST/mInx-en.git 

* cd/mlnx-en/ofed-scripts 

e  /set irq affinity bynode.sh «numa node id» <interface-name> 
* Set irq affinity -x local <interface-name> 


Enabling RSS and TPS 


NOTE: RSS/TPS is disabled 
CPS (2 nginx instances) with HT 
nginx workers 
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Enabling RSS and TPS 


NOTE: RSS/TPS is enabled 
CPS (2 nginx instances) with HT 
nginx workers 
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summary 


Deploy two nginx instances 

Using additional nginx configuration directives 
Linux sysctl parameter tuning 

Setting IRQ affinity using RSS and TPS 

Consult appendix for additional information and 
performance tips 


NGINX 


Thank you 


Contact information here 


amir.rawdat@nginx.com 


Appendix 


Client Traffic Script 


taskset -c 0-21,44-65 wrk -t 44 -c 1000 -d 180s -H 'Connection: Close' https:// 
10.10.16.10:443/$1 >> output.txt 8 

taskset -c 22-43,66-87 wrk -t 44 -c 1000 -d 180s -H "Connection: Close" https:// 
10.10.11.23:443/51 >> output1.txt 8 


## $1 is the requested static file size 
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Reverse proxy (Instance 1) 


user root; 

worker. processes 48 ; 

worker. cpu affinity auto 
000000000000000000000000111111111111111111111111000000000000000000000000111111111111111111111111; 

worker rlimit nofile 1024000; 

error. log /home/ubuntu/access.error crit; 

events ( 
worker. connections 1000000; 

} 


http { 
access. log off; 
keepalive timeout 315; 
keepalive requests 10000000; 


sendfile on; 
tcp. nopush on; 
tcp. nodelay on; 
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Reverse proxy (Instance 1) 


server ( 
listen 10.10.10.18:443 ssl backlog-102400 reuseport; 
ssl certificate /etc/ssl/certs/nginx.pem; 
ssl certificate key /etc/ssl/private/nginx.key; 
ssl session cache off; 
ssl session tickets off; 


location / ( 
proxy http. version 1.1; 
proxy.set header Connection ""; 
proxy.pass http://webserver. 0; 


} 
} 
upstream webserver 8 4 
server 10.10.10.11:80; 
keepalive 200; 
) 
} 
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Reverse proxy (Instance 2) 


user root; 

worker. processes 48 ; 

worker. cpu affinity auto 
111111111111111111111111000000000000000000000000111111111111111111111111000000000000000000000000; 

worker rlimit nofile 1024000; 

error. log /home/ubuntu/access.error crit; 

events ( 
worker. connections 1000000; 

} 


http { 
access. log off; 
keepalive timeout 315; 
keepalive requests 10000000; 


sendfile on; 
tcp. nopush on; 
tcp. nodelay on; 
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Reverse proxy (Instance 2) 


server ( 
listen 10.10.15.9:443 ssl backlog-102400 reuseport; 
ssl certificate /etc/ssl/certs/nginx.pem; 
ssl certificate key /etc/ssl/private/nginx.key; 
ssl session cache off; 
ssl session tickets off; 


location / ( 
proxy http. version 1.1; 
proxy.set header Connection ""; 
proxy pass http://webserver. 1; 
} 
} 
upstream webserver 1 4 
server 10.10.15.12:80; 
keepalive 200; 
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